package org.shj.demo;

import com.hankcs.hanlp.classification.classifiers.IClassifier;
import com.hankcs.hanlp.classification.classifiers.NaiveBayesClassifier;
import com.hankcs.hanlp.classification.corpus.FileDataSet;
import com.hankcs.hanlp.classification.corpus.IDataSet;
import com.hankcs.hanlp.classification.corpus.MemoryDataSet;
import com.hankcs.hanlp.classification.statistics.evaluations.Evaluator;
import com.hankcs.hanlp.classification.statistics.evaluations.FMeasure;
import com.hankcs.hanlp.classification.tokenizers.BigramTokenizer;
import com.hankcs.hanlp.classification.tokenizers.HanLPTokenizer;
import com.hankcs.hanlp.classification.tokenizers.ITokenizer;
import com.hankcs.hanlp.tokenizer.NLPTokenizer;
import org.shj.tokenizer.CustomerNLPTokenizer;

/**
 * @author Shen Huang Jian
 * @date 2020-09-29 16:25
 */
public class ClassifyMeasureDemo {

    public static void main(String[] args){

        try {
            //String corpusPath = "D:\\source\\hanLp\\data\\test\\weibo-classification";
            String corpusPath = "D:\\source\\hanLp\\data\\test\\ChnSentiCorp情感分析酒店评论";

            // 支持不同的ITokenizer，详见源码中的文档
//            ITokenizer tokenizer = new BigramTokenizer();
            ITokenizer tokenizer = new HanLPTokenizer();

//            ITokenizer tokenizer = new CustomerNLPTokenizer();

            // FileDataSet省内存，可加载大规模数据集
            FileDataSet fileDataSet = new FileDataSet();
            fileDataSet.setTokenizer(tokenizer);

            // 前90%作为训练集
            double trainRate = 0.9;
            IDataSet trainingCorpus = fileDataSet.load(corpusPath, "UTF-8", trainRate);

            IClassifier classifier = new NaiveBayesClassifier();
            classifier.train(trainingCorpus);

            // 后10%作为测试集
            double testRate = trainRate - 1;
            IDataSet testingCorpus = new MemoryDataSet(classifier.getModel()).
                    load(corpusPath, "UTF-8", testRate);
            // 计算准确率
            FMeasure result = Evaluator.evaluate(classifier, testingCorpus);
            System.out.println(result);
        }catch (Exception e){
            e.printStackTrace();
        }
    }

}
